Demo for MPII Reader

In [1]:
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime, date, time
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import cv2
import math
import scipy.misc
import hashlib
from os import listdir
from os.path import isfile, join
import os
import random
In [2]:
# notebook in src/training => set the project folder as root
# /!\ execute once
os.chdir("../../")

Import mpii reader and drawer

In [3]:
from src.training import mpii
In [4]:
from src.utils.drawer import Drawer
In [5]:
drawer = Drawer()

draw bounding box + 2d skeleton of all persons in a given : mpiiDataset, entryId

In [6]:
def drawImage(mpiiDataset, entryId, thickness=8):
    
    img = mpiiDataset.getImage(entryId)
    
    for personId in range(mpiiDataset.getTotalPeopleOn(entryId)):
        
        bbox =  mpiiDataset.getBoundingBox(entryId, personId)
        bbox = {'minX':bbox[0], 'maxX':bbox[1], 'minY':bbox[2], 'maxY':bbox[3]}
        
        convertedJoints = []
        for joint in mpiiDataset.getJoints(entryId, personId):
            convertedJoints.append([int(joint['x']), int(joint['y'])])
        convertedJoints = np.array(convertedJoints)
        img = drawer.drawBBox(img, bbox)
        img = drawer.draw_2d_skeleton(img, convertedJoints, thickness)
    
    plt.imshow(img)
    plt.show()

Load the mpii dataset

In [7]:
# images
imgFolder = "dataset/mpii/data/images"
# previous and next frames (not annotated)
videoFolder = "dataset/mpii/data/videos"
# images annotations
imgAnnot = 'dataset/mpii/data/metadata/mpii_human_pose_v1_u12_1.mat'
# videos annotations (link annotated_img => previous_and_next_frames)
videoAnnot = 'dataset/mpii/data/metadata/mpii_human_pose_v1_sequences_keyframes.mat'
In [8]:
dataset = mpii.Reader.build(imgFolder,videoFolder, imgAnnot, videoAnnot)
dataset.size()
Out[8]:
17408
In [9]:
drawImage(dataset, 10)
In [10]:
dataset.getMetadata(10)
Out[10]:
{'activity': 'sitting, talking in person, on the phone, computer, or text messaging, light effort',
 'category': 'miscellaneous',
 'framePath': '030424224/00000051.jpg',
 'imageShape': (480, 854, 3),
 'imgName': '030424224.jpg',
 'initTotalUnannotated': 0,
 'init_total_people': 2,
 'people': [{'joints': [{'x': 324, 'y': 406},
    {'x': 322, 'y': 291},
    {'x': 293, 'y': 262},
    {'x': 355, 'y': 267},
    {'x': 401, 'y': 292},
    {'x': 394, 'y': 404},
    {'x': 324, 'y': 265},
    {'x': 334, 'y': 170},
    {'x': 338.6651, 'y': 157.2087},
    {'x': 360.3349, 'y': 97.7913},
    {'x': 363, 'y': 167},
    {'x': 319, 'y': 205},
    {'x': 298, 'y': 168},
    {'x': 369, 'y': 172},
    {'x': 390, 'y': 221},
    {'x': 398, 'y': 179}],
   'total_joints': 16},
  {'joints': [{'x': 476, 'y': 407},
    {'x': 476, 'y': 295},
    {'x': 510, 'y': 264},
    {'x': 564, 'y': 265},
    {'x': 581, 'y': 303},
    {'x': 575, 'y': 412},
    {'x': 537, 'y': 265},
    {'x': 535, 'y': 175},
    {'x': 534.9093, 'y': 172.7919},
    {'x': 532.0907, 'y': 104.2081},
    {'x': 508, 'y': 183},
    {'x': 492, 'y': 238},
    {'x': 493, 'y': 173},
    {'x': 577, 'y': 176},
    {'x': 591, 'y': 235},
    {'x': 556, 'y': 190}],
   'total_joints': 16}]}

Filter bad examples

mpii.py MetaMapper can either :

  1. modify the meta
  2. filter per person metadata
  3. remove the all meta (and so the image)

mpii.py ImageMapper :

  1. lazily transform img=>img

new dataset can be created easily with a mapper

some images has unannotated persons

In [11]:
# this is a metaMapper function : meta => 
# if returned meta == None :then: filtered
# here : keep only images where some people are not annotated

metaMapper = lambda meta: meta if meta["initTotalUnannotated"] > 0 else None

somePersonNotAnnotateddataset = dataset.mapMetadata(metaMapper)

print(somePersonNotAnnotateddataset.size())

for i in range(somePersonNotAnnotateddataset.size()):
    drawImage(somePersonNotAnnotateddataset, i)
    if(i>10):
        break

127
In [11]:
# filter the examples above : 
metaMapper = lambda meta: meta if meta["initTotalUnannotated"] == 0 else None
dataset = dataset.mapMetadata(metaMapper)
dataset.size()
Out[11]:
17281

some categories are less adapated for the task

In [13]:
def getFilterFor(category, activity):
    def metaMapper(meta):
        isOk = meta["category"] != None 
        isOk = isOk and category in meta["category"]
        isOk = isOk and meta["activity"] != None 
        isOk = isOk and activity in meta["activity"]
        return meta if isOk else None
    return metaMapper


tmpDataset = dataset.mapMetadata(getFilterFor("music","orchestra"))
print(tmpDataset.size())
for i in range(tmpDataset.size()):
    drawImage(tmpDataset, i)
    if i>5:
        break;

tmpDataset = dataset.mapMetadata(getFilterFor("","bicycling, racing"))
print(tmpDataset.size())
for i in range(tmpDataset.size()):
    drawImage(tmpDataset, i)
    if i>5:
        break;
        
tmpDataset = dataset.mapMetadata(getFilterFor("water","canoe"))
print(tmpDataset.size())
for i in range(tmpDataset.size()):
    drawImage(tmpDataset, i)
    if i>5:
        break;
    
tmpDataset = dataset.mapMetadata(getFilterFor("water","swim"))
print(tmpDataset.size())
for i in range(tmpDataset.size()):
    drawImage(tmpDataset, i)
    if i>5:
        break;
    

12

153

216

234
In [12]:
# filter the examples above : 
def filterCategory(activity):
    return lambda meta : meta if (meta["activity"] != None and activity not in meta["activity"]) else None

dataset = dataset.mapMetadata(filterCategory("orchestra"))
dataset = dataset.mapMetadata(filterCategory("bicycling, racing"))
dataset = dataset.mapMetadata(filterCategory("canoe"))
dataset = dataset.mapMetadata(filterCategory("swim"))
dataset.size()
Out[12]:
16549

some person has joints outside the scene ( joint position set to -1) => remove all

In [13]:
dataset = dataset.mapMetadata(mpii.MetaMapper.filterAllJointsInsideScene())
dataset.size()
Out[13]:
11340

The woman isnt labeled anymore because of missing ankle

In [14]:
drawImage(dataset, 6)

filter images with new unannotated people

In [17]:
dataset = dataset.mapMetadata(mpii.MetaMapper.containsUnannotatedPeople())
drawImage(dataset, 6)
dataset.size()
Out[17]:
9642

Filter images with too small people

all persons in the image has to have a bbox_area >= totalpix_given_as_arg

In [18]:
print(dataset.getBoundingBox(0,0))
print(dataset.getBoundingBox(0,1))
drawImage(dataset, 0)
tmpDataset = dataset.mapMetadata(mpii.MetaMapper.filterSmallPeople(400*150))
drawImage(tmpDataset, 0)

(553, 695, 108, 394)
(830, 1013, 174, 403)

Filter entry per image name

some images has still unannotated people. A pre train object detection could be used to detect such cases. This filter remove all the images contained in the argument (iterable of imgName)

In [19]:
imgNames = []
for i in range(8):
    print(i)
    if(i<5):
        imgNames.append(dataset.getImageName(i))
    drawImage(dataset, i)
    
tmpDataset = dataset.mapMetadata(mpii.MetaMapper.filterImageNameIn(imgNames))

print("======================= FIlTERED ===============================")
for i in range(3):
    drawImage(tmpDataset, i)

0

1

2

3

4

5

6

7

======================= FIlTERED ===============================

 Filter overlapped people

filter metadata when people overlap : filter if float(intersectionArea)/min(person1_area, person2_area) <= givenRate

In [20]:
# from the tmp metadata above

# max 20% of shared space bewteen people
sharedPrecent = 0.2
drawImage(tmpDataset, 0)
drawImage(tmpDataset, 1)
drawImage(tmpDataset, 2)
tmpDataset2 = tmpDataset.mapMetadata(mpii.MetaMapper.filterOverlappedPeople(sharedPrecent))
print("======================= FIlTERED ===============================")
drawImage(tmpDataset2, 0)
drawImage(tmpDataset2, 1)
drawImage(tmpDataset2, 2)

======================= FIlTERED ===============================

Resize dataset to fit 480x640 resolution

In [22]:
# demo on imgId = 25
drawImage(dataset,25)

crop image in the center to get the 480/640 ratio

can crop 480/640 in the center

In [27]:
cropCenter = 1
datasetResized_center = dataset.mapMetadata(mpii.MetaMapper.cropImageToRatio(480/640.0, cropCenter))
datasetResized_center = datasetResized_center.mapImage(mpii.ImageMapper.cropImageToRatio(480.0/640.0, cropCenter))
In [28]:
drawImage(datasetResized_center,25)

can also crop 480/640 left/right

In [29]:
# can also crop left/right
left = 0
right = 2
datasetResized_left = dataset.mapMetadata(mpii.MetaMapper.cropImageToRatio(480/640.0, left))
datasetResized_left = datasetResized_left.mapImage(mpii.ImageMapper.cropImageToRatio(480.0/640.0, left))

datasetResized_right = dataset.mapMetadata(mpii.MetaMapper.cropImageToRatio(480/640.0, right))
datasetResized_right = datasetResized_right.mapImage(mpii.ImageMapper.cropImageToRatio(480.0/640.0, right))
In [30]:
drawImage(datasetResized_left,25)
drawImage(datasetResized_right,25)

Resize image to webcam resolution : 640x480

In [31]:
# demo on imgId = 10 : initial img
drawImage(dataset,10)
# cropped center to get the right height/width ratio
drawImage(datasetResized_center,10)

Apply the resize to get 640x480 resolution

In [32]:
datasetResized_center = datasetResized_center.mapMetadata(mpii.MetaMapper.resizeTo(640, 480))
datasetResized_center = datasetResized_center.mapImage(mpii.ImageMapper.resizeTo(640, 480))

The image has now 640x480 without distortion

In [33]:
drawImage(datasetResized_center,10)

 Clean metadata after resize

after resizing, joint position keep consistent, however they could stand outside the image

In [34]:
drawImage(datasetResized_center,26)
In [35]:
cleanedDataset = datasetResized_center.mapMetadata(mpii.MetaMapper.filterAllJointsInsideScene())
In [36]:
# note : one image was filter : imgId=26 before : imgId=25 here
drawImage(cleanedDataset,25)
In [37]:
# as before i can filter all images that contains unannotated people (like the one above)
print(cleanedDataset.size())
cleanedDataset = cleanedDataset.mapMetadata(mpii.MetaMapper.containsUnannotatedPeople())
print(cleanedDataset.size())
drawImage(cleanedDataset,25)

9243
8388

Data Augmentation

Add Symetry

In [38]:
drawImage(cleanedDataset,21)
In [39]:
# sym on image
symDataset = cleanedDataset.mapImage(mpii.ImageMapper.verticalFlip())
# sym on meta
symDataset = symDataset.mapMetadata(mpii.MetaMapper.verticalFlip())
In [40]:
# TODO : imageShape constante : do recursive call to child 
drawImage(symDataset,21)

Add random saturation/luminosity variation

In [42]:
# in my opinion : valueCoeff=0.3, saturationCoeff=0.3 enough for slight modif
symDatasetVar = symDataset.mapImage(mpii.ImageMapper.randomValueSaturation(valueCoeff=0.8, saturationCoeff=0.8))
cleanedDatasetVar = cleanedDataset.mapImage(mpii.ImageMapper.randomValueSaturation(valueCoeff=0.8, saturationCoeff=0.8))
In [43]:
for _ in range(10):
    drawImage(symDatasetVar,48)
    drawImage(cleanedDatasetVar,48)

Add rotation

In [44]:
# symDatasetVar
# cleanedDatasetVar
for i in range(5):
    
    degreeRot = random.random()*360
    
    # rotate sym dataset
    symDatasetVarRotated = symDatasetVar.mapImage(mpii.ImageMapper.rotate(degreeRot))
    symDatasetVarRotated = symDatasetVarRotated.mapMetadata(mpii.MetaMapper.rotate(degreeRot))
    
    # rotate dataset
    cleanedDatasetVarRotated = cleanedDatasetVar.mapImage(mpii.ImageMapper.rotate(degreeRot))
    cleanedDatasetVarRotated = cleanedDatasetVarRotated.mapMetadata(mpii.MetaMapper.rotate(degreeRot))
    
    drawImage(symDatasetVarRotated, 48)
    drawImage(cleanedDatasetVarRotated, 48)

 Add Transpose

In some case we do not want the 0 padding added by a random rotation (the network could for example use the 0 padding as an indicator for detecting weird rotated pose)

In [45]:
transposeDataset = symDataset.mapImage(mpii.ImageMapper.transpose())
transposeDataset = transposeDataset.mapMetadata(mpii.MetaMapper.transpose())
In [47]:
drawImage(symDataset, 10)
drawImage(transposeDataset,10)